/*
 * @(#)ContentExtractor.java	0.4 12/28/04
 *
 * Copyright 2004 Fabio Gasparetti. All rights reserved.
 */

package org.tabularium.text.html;

import java.io.*;
import java.net.*;
import java.util.*;
import java.util.regex.*;

import javax.swing.JFrame;
import javax.swing.text.html.*;
import javax.swing.text.html.HTMLEditorKit.*;
import javax.swing.text.html.parser.*;

import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.standard.*;
import org.htmlparser.*;
import org.htmlparser.util.Translate;
import org.htmlparser.lexer.*;
import org.htmlparser.tags.*;
import org.htmlparser.visitors.*;
import org.htmlparser.beans.*;
import org.htmlparser.nodes.*;
import org.tabularium.net.UrlNormalizer;

/**
 * Some utilities for html page processing.
 * 
 * @author Fabio Gasparetti
 * @version 0.4, 12/28/04
 */

public class HtmlParserContentExtractor extends ContentExtractor {
	
	protected boolean imageAlts = true;

	protected boolean linkTitles = true;
	
	protected HashSet htmlNotFlowBreakSet = new HashSet();	

	public HtmlParserContentExtractor() {
		for (int i = 0; i < htmlNotFlowBreak.length; i++) {
			this.htmlNotFlowBreakSet.add(htmlNotFlowBreak[i]);
		}		
	}
	/**
	 * Extracts the text from an html page.
	 * 
	 * Character references and character entity references to unicode
	 * characters.
	 * 
	 * Note: some of javascript code might be present in the extracted content
	 * if it contains HTML tags.
	 * 
	 */
	public String extractText(String url, String htmltext) throws Exception {
		Lexer lexer;
		org.htmlparser.Parser parser;
		String ret = null;

		lexer = new Lexer(htmltext);
		parser = new org.htmlparser.Parser(lexer);
		TextVisitor visitor = new TextVisitor();
		visitor.imageAlts = imageAlts;
		visitor.linkTitles = linkTitles;
		visitor.htmlNotFlowBreakSet = htmlNotFlowBreakSet;
		parser.visitAllNodesWith(visitor);
		ret = visitor.getExtractedText();
		return ret;
	}

	/**
	 * Note: retrieving images' ALTs could break language sentences
	 * 
	 * @param on
	 */
	public void retrieveImageAlts(boolean on) {
		imageAlts = on;
	}

	/**
	 * Note: retrieving links' TITLEs could break language sentences
	 * 
	 * @param on
	 */
	public void retrieveLinkTitles(boolean on) {
		linkTitles = on;
	}

	// for debugging
	public static void main(String[] args) {
		try {
			InputStream is = new FileInputStream("prova.html");
			BufferedReader reader = new BufferedReader(
					new InputStreamReader(is));

			String line;
			StringBuffer doc = new StringBuffer();
			while ((line = reader.readLine()) != null) {
				doc.append(line + "\n");
			}
			is.close();
			HtmlParserContentExtractor ce = new HtmlParserContentExtractor();
			System.out.println("extracted text: "
					+ ce.extractText(".", doc.toString()));
		} catch (Exception ex) {
			System.out.println(ex.toString());
		}
	}
}

/* HtmlParser visitor for text extraction. See extractHtmlText(). */
class TextVisitor extends StringBean {
	StringBuffer buffer = new StringBuffer();

	boolean imageAlts = false;

	boolean linkTitles = false;
	
	HashSet htmlNotFlowBreakSet = null;	

	public TextVisitor() {
		super();
	}

	public String getExtractedText() {
		return buffer.toString();
	}

	public void visitTag(Tag tag) {
		if (breaksSentence(tag))
			buffer.append(" ");

		if (tag instanceof TitleTag) {
			String s = ((TitleTag) tag).getTitle();
			if (s != null)
				buffer.append(s);
		} else if (linkTitles && (tag instanceof LinkTag)) {
			String s = tag.getAttribute("TITLE");			
			// warning: retrieving links' TITLEs could break language sentences
			if (s != null)
				buffer.append(" " + Translate.decode(s) + " ");
		} else if (imageAlts && (tag instanceof ImageTag)) {
			String s = tag.getAttribute("ALT");
			// warning: retrieving images' ALTs could break language sentences
			if (s != null)
				buffer.append(" " + Translate.decode(s) + " ");
		}

		if (breaksSentence(tag))
			buffer.append(" ");
	}

	public boolean breaksSentence(Tag t) {
		boolean ret = true;
		if (htmlNotFlowBreakSet.contains(t.getTagName()))
				ret = false;
		return ret || t.breaksFlow();
	}

	public void visitStringNode(Text stringNode) {
		Node parent = stringNode.getParent();
		if ((parent != null)
				&& (parent instanceof org.htmlparser.tags.ScriptTag)) {
			return;
		}
		if ((parent != null)
				&& (parent instanceof org.htmlparser.tags.StyleTag)) {
			return;
		}
		if (!mIsScript && !mIsStyle) {
			String text = stringNode.getText();
			if (!mIsPre) {
				translateAppend(buffer, text);
			} else
				buffer.append(text);
		}
	}
	
	public void translateAppend(StringBuffer sb, String text) {
		text = Translate.decode(text);
		if (getReplaceNonBreakingSpaces())
			text = text.replace('\u00a0', ' ');
		if (getCollapse())
			collapse(sb, text);
		else
			sb.append(text);	
	}
}
